import requests
import os
import sys
import json
import datetime
import re
import logging
import time
import math
pwd = os.getcwd()
sys.path.append(pwd)
# from proxy import Proxy
from lxml import etree
# gov数据库
# from sql import sql_gov_model
from sql import sql_gov_model


class GovStcsmsh(object):
    url = 'http://jmjh.miit.gov.cn/cmsadmin/front/channel/list/search.do'
    headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
        'Accept-Encoding': 'gzip, deflate',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Connection': 'keep-alive',
        'Cookie': 'Hm_lvt_af6f1f256bb28e610b1fc64e6b1a7613=1589505096; Hm_lpvt_af6f1f256bb28e610b1fc64e6b1a7613=1589509587; waf_cookie=c46cfe09-fae5-42a0d3d4f2302142060fc92ce0476be99b2c; TOKEN=99a5f3e785c94438821bfe20c22d5023; bg6=6|ABuC3; bg12=19|ABuC3',
        'Host': 'jmjh.miit.gov.cn',
        'Referer': 'http://jmjh.miit.gov.cn/cmsadmin/front/channel/list/search.do?pageNo=2&pageSize=20&keyword=%E5%85%AC%E5%91%8A&channelId=&keywordAnd=',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36',
    }
    base_dir = './'
    def __init__(self, keyword):
        # self.proxy = Proxy().get_proxy()
        self.keyword = keyword
        self.yestoday = self.get_yestoday().replace(':', '-')
        save_dir = self.base_dir + self.yestoday
        # self.folder_exist(save_dir)
        self.save_file = save_dir + '/' + keyword + '.csv'
        self.params = {
            'pageNo': '1', # 当前页数
            'pageSize': '20', # 每页显示的数量
            'keyword': keyword, # 搜索的关键词
            'channelId': '', # 选择公告类型，为空：表示搜索所有
            'keywordAnd': '', # 在搜索结果中再次查询
        }
    # 获取昨天的年月日
    def get_yestoday(self):
        yesterday = datetime.datetime.today() + datetime.timedelta(-1)
        str_yesterday = datetime.datetime.strftime(yesterday, "%Y:%m:%d")
        return str_yesterday

    # 请求网页,获取html
    def request(self):
        response = requests.get(url=self.url, params=self.params, headers=self.headers, timeout=20)
        self.response = response.text
        # with open('jmjh_miit.html', 'w') as f:
        #     f.write(self.response)
        max_page = self.get_max_page()
        # print(max_page)
        current_page = 1
        while current_page <= max_page:
            self.params['pageNo'] = current_page
            self.response = requests.get(url=self.url, params=self.params, headers=self.headers, timeout=20).text
            self.page_parse()
            current_page += 1

    # 获取最大页数
    def get_max_page(self):
        try:
            record = re.search('totalPage: (\d+),', self.response).group(1)
        except AttributeError:
            record = 0
        return int(record)
    
    # 获取招标信息
    def page_parse(self):
        html = etree.HTML(self.response)
        list_content = html.xpath('/html/body/div[@class="content"]/div[@class="noticeList-box banner"]/ul/li') # 一页的所有记录
        # 如果list_content长度为0,说明当前页没有一条记录
        if len(list_content) == 0:
            return None
        for current_content in list_content:
            str_type = current_content.xpath('span[1]/text()')[0] # 公告类型
            url = current_content.xpath('a/@href')[0] # 链接
            if 'http' not in url:
                url = 'http://jmjh.miit.gov.cn' + url
            title = current_content.xpath('a')[0].xpath('string(.)').strip().replace(' ', '').replace(',', '，').replace('\n', '') # 标题
            str_time = current_content.xpath('span[2]/text()')[0]
            str_time = datetime.datetime.strptime(str_time, '%Y-%m-%d')  # 发表时间
            
            print(url, title, str_type, str_time)
            gov = sql_gov_model.Gov(website='国家军民融合公共服务平台', keyword=self.keyword, title=title, url=url, str_time=str_time, str_type=str_type, desc='')
            sql_gov_model.insert_gov(gov)
    
    # 向文件写内容
    def write_file(self, content):
        with open(self.save_file, 'a') as f:
            f.write(content)

    def folder_exist(self, dir_path):
        '''
        1. 作用:判断文件夹路径是否存在,不存在则创建
        2. 参数:dir_path:文件夹路径
        3. 返回值:None
        '''
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)

if __name__ == '__main__':
    list_keywords = ['人工智能', '人才', '职称', '项目', '专项', '指南', '申报', '奖励']
    for keywords in list_keywords:
        gov_stcsmsh = GovStcsmsh(keywords)
        gov_stcsmsh.request()
